/*
**
** 
**      Extractor 7.2
**
**      Copyright (c) 1996-2001
**
**      National Research Council of Canada
**
**
**
**
**
**      Peter Turney
**      Benoit Gagnon
**      Jerome Mathieu
**      Selim Levy
**      Joontae Kim
**
**      November 26, 2001
**
**
**
**
**      test_api.c
**
**
**
**
**
**      test the extractor API
**
*/

/*
**      include standard stuff
*/

#include <stdio.h>
#include <string.h>
#include <time.h>

/*
**      include the header for the API DLL
*/

#include "extr_api.h"

/*
**      Windows versus Unix
*/

#if (defined(WIN32) && defined(_CONSOLE))
#define       WINFLAG             1
#else
#define       WINFLAG             0
#endif

/*
**      length of input buffer for reading text from file
*/

#define       MAX_BUF_LEN         32768

/*
**      length of string for input from user from standard
**      input
*/

#define       IN_LEN              256

/*
**      language codes
*/

#define       LAN_UNKNOWN         0
#define       LAN_ENGLISH         1
#define       LAN_FRENCH          2
#define       LAN_JAPANESE        3
#define       LAN_GERMAN          4
#define       LAN_SPANISH         5
#define       LAN_KOREAN          6

/*
**      minimum and maximum values for LanguageID
*/

#define       MIN_LANG_ID         0
#define       MAX_LANG_ID         6

/*
**      character encodings
*/

#define       ENC_LAT             0
#define       ENC_DOS             1
#define       ENC_UNI             2
#define       ENC_SJIS            3
#define       ENC_JIS             4
#define       ENC_EUC             5
#define       ENC_KSC             6
#define       ENC_JOHAP           7

/*
**      minimum and maximum values for  CharCodeID
*/

#define       MIN_CHAR_ID         0
#define       MAX_CHAR_ID         7

/*
**      types of matching for ExtrAddGoPhrase
*/

#define       MATCH_DEFAULT                 0
#define       MATCH_EXACT_CASE              1
#define       MATCH_WHOLE_WORD              2
#define       MATCH_EXACT_WIDTH             4
#define       MATCH_EXACT_ACCENT            8
#define       MATCH_MULTIPLE_SUBSTRINGS    16
#define       MATCH_IGNORE_SPACES          32

/*
**      types of document properties
*/

#define       PROP_WORDS          1
#define       PROP_NONSTOP        2
#define       PROP_WHOLE          3

/*
**      name of file for HTML output
*/

#define       HTML_FILE_NAME      "test_api.html"

/*
**      main procedure
*/

int main(void)
{
  /*
  **    local variable declarations
  */

  int    LanguageID    = 0;
  int    InCharCodeID  = 0;
  int    OutCharCodeID = 0;
  double DesiredNumber = 7; 
  int    HighlightType = 31;

  int    PropID;
  int    PropValue;

  void  *StopMemory;
  void  *DocumentMemory;

  char   DocumentBuffer[MAX_BUF_LEN];
  int    BufferLength;

  int    PhraseListSize;
  int    PhraseIndex;
  char  *Phrase;
  double Score;
  int    OutputDevice;
  int    HighlightListSize;
  int    HighlightIndex;
  char  *Highlight;

  int    ErrorCode;
  char  *ErrorMessage;

  FILE  *text_file;
  FILE  *out_file;
  long   file_length;
  char   file_name[IN_LEN];
  char   filter_html[IN_LEN];
  char   filter_mail[IN_LEN];
  char   language_string[IN_LEN];
  char   num_phrase_string[IN_LEN];
  char   code_string[IN_LEN];
  char   output_device[IN_LEN];
  char   go_phrase[IN_LEN];

  time_t t_start, t_stop;
  int    run_time;

  int    Windows = WINFLAG;

  /*
  **    print a nice introductory message to the standard output
  */

  printf(
    "\n"
    "                               EXTRACTOR 7.2\n"
    "\n"
    "                            Copyright (c) 1996-2001\n"
    "                        National Research Council Canada\n"
    "\n"
    "               This demonstration software expires: June 30, 2002.\n"
    "               For more information, contact: peter.turney@nrc.ca.\n"
    "\n"
    "EXTRACTOR is software for automatically summarizing text, developed by the\n"
    "Interactive Information Group of the National Research Council of Canada.\n"
    "EXTRACTOR takes a text file as input and generates a list of key words and a\n"
    "list of key sentences as output. This version of EXTRACTOR is designed for\n"
    "use with English, French, Japanese, German, Spanish, and Korean text. The\n"
    "purpose of this demonstration software is to allow you to test the value of the\n"
    "technology. If you find the technology useful, you may obtain a license to the\n"
    "source code, which you may then include in your own software.\n"
    "\n"
    "Please enter the name of the text file. A blank entry will cause the program to\n"
    "exit. The file may contain plain ASCII text, HTML, or e-mail. If the file is\n"
    "not in the current working directory, you can enter the full path name.\n"
    "\n");

  /*
  **    ask user for input file name
  */

  printf("File Name > ");
  fgets(file_name, IN_LEN, stdin);
  file_name[strcspn(file_name, "\n\r")] = '\0';

  if (file_name[0] == '\0') {
      printf("\n\nExiting, as requested. \n\n");
      return 0;
  }

  /*
  **    open input file for reading
  **
  **    - open file in binary mode, in case file is in Unicode
  */

  text_file = fopen(file_name, "rb");

  if (text_file == NULL) {
    printf("\n\nERROR: cannot open file \"%s\". \n\n", file_name);
    return 0;
  }

  /*
  **    determine the length of the input file
  **
  **    - remember to return to the top of the file
  */

  fseek(text_file, 0L, SEEK_END);
  file_length = ftell(text_file);
  rewind(text_file);

  if (file_length == 0L) {
    printf("\n\nERROR: empty file \"%s\". \n\n", file_name);
    return 0;
  }

  /*
  **    set up the stop word and stop phrase lists
  */

  ErrorCode = ExtrCreateStopMemory(&StopMemory);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    add an English stop word (just to show how it's done)
  */

  LanguageID   = LAN_ENGLISH;
  InCharCodeID = ENC_LAT;
  ErrorCode    = ExtrAddStopWord(StopMemory, LanguageID, 
                   InCharCodeID, (void *) "the");

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    add an English stop phrase (just to show how it's done)
  */

  LanguageID   = LAN_ENGLISH;
  InCharCodeID = ENC_LAT;
  ErrorCode    = ExtrAddStopPhrase(StopMemory, LanguageID, 
                   InCharCodeID, (void *) "group");

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    add a Japanese stop phrase (just to show how it's done)
  */

  LanguageID   = LAN_JAPANESE;
  InCharCodeID = ENC_SJIS;
  ErrorCode    = ExtrAddStopPhrase(StopMemory, LanguageID, 
                   InCharCodeID, (void *) "\x8d\xc4\x93\x78");

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    add a Korean stop word (just to show how it's done)
  */

  LanguageID   = LAN_KOREAN;
  InCharCodeID = ENC_KSC;
  ErrorCode    = ExtrAddStopWord(StopMemory, LanguageID, 
                   InCharCodeID, (void *) "\xb1\xd7\xb3\xe0");

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    add a Korean stop phrase (just to show how it's done)
  */

  LanguageID   = LAN_KOREAN;
  InCharCodeID = ENC_KSC;
  ErrorCode    = ExtrAddStopPhrase(StopMemory, LanguageID, 
                   InCharCodeID, (void *) "\xb1\xd7\xb3\xe0");

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    create the document memory
  */

  ErrorCode = ExtrCreateDocumentMemory(&DocumentMemory);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    ask user for language
  */

  printf(
    "\n"
    "Select a language for the document or let EXTRACTOR guess. Enter a\n"
    "blank line for the default value (0 = Automatic) or enter a number.\n"
    "\n"
    "   0 = Automatic    Automatically detect English, French, German, or Spanish.\n"
    "   1 = English      Force EXTRACTOR to interpret the document as English.\n"
    "   2 = French       Force EXTRACTOR to interpret the document as French.\n"
    "   3 = Japanese     Force EXTRACTOR to interpret the document as Japanese.\n"
    "   4 = German       Force EXTRACTOR to interpret the document as German.\n"
    "   5 = Spanish      Force EXTRACTOR to interpret the document as Spanish.\n"
    "   6 = Korean       Force EXTRACTOR to interpret the document as Korean.\n" 
    "\n"
    "Note that automatic detection is not currently available for Japanese and\n"
    "Korean.\n"
    "\n");

  printf("Language > ");
  fgets(language_string, IN_LEN, stdin);
  language_string[strcspn(language_string, "\n\r")] = '\0';

  if (language_string[0] == '\0') {
    printf("\nEXTRACTOR will automatically detect the language.\n");
    LanguageID = LAN_UNKNOWN;
  }
  else {
    ErrorCode = sscanf(language_string, "%d", &LanguageID);
    if (ErrorCode != 1) {
      printf("\n\nERROR: your input is not a valid number.\n\n");
      return 0;
    }
    if ((LanguageID < MIN_LANG_ID) || (LanguageID > MAX_LANG_ID)) {
      printf("\n\nERROR: your input is out of range.\n\n");
      return 0;
    }
    ErrorCode = ExtrSetDocumentLanguage(DocumentMemory, LanguageID);
    if (ErrorCode > 0) {
      ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
      printf("\n\n%s\n\n", ErrorMessage);
      return 0;
    }
    printf("\nLanguage is set to %d. \n", LanguageID);
  }

  /*
  **    ask user whether HTML filtering is required
  */

  printf(
    "\n"
    "If the input file contains HTML (HyperText Markup Language), then the markup\n"
    "commands should be removed. Would you like to run an HTML filter on the input\n"
    "text file? Enter a blank line for the default value (no HTML filter) or enter\n"
    "\"f\" to filter.\n"
    "\n");

  printf("HTML Filter > ");
  fgets(filter_html, IN_LEN, stdin);
  filter_html[strcspn(filter_html, "\n\r")] = '\0';

  if (filter_html[0] == '\0') {
    printf("\nNo HTML filtering. \n");
  }
  else {
    ErrorCode = ExtrActivateHTMLFilter(DocumentMemory, StopMemory);
    if (ErrorCode > 0) {
      ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
      printf("\n\n%s\n\n", ErrorMessage);
      return 0;
    }
    printf("\nHTML filter activated. \n");
  }

  /*
  **    ask user whether mail filtering is required
  */

  if ((LanguageID == LAN_JAPANESE) || (LanguageID == LAN_KOREAN)) {
    printf(
      "\n"
      "Note: The e-mail filtering option is not currently available for Japanese and\n"
      "Korean.\n"
      "\n");
  }
  else {
    printf(
      "\n"
      "If the input file contains e-mail, then the mail headers (\"From:\", \"To:\",\n"
      "etc.) should be removed. Would you like to run a mail filter on the input\n"
      "text file? Enter a blank line for the default value (no mail filter) or enter\n"
      "\"f\" to filter.\n"
      "\n");
    
    printf("Mail Filter > ");
    fgets(filter_mail, IN_LEN, stdin);
    filter_mail[strcspn(filter_mail, "\n\r")] = '\0';
    
    if (filter_mail[0] == '\0') {
      printf("\nNo mail filtering. \n");
    }
    else {
      ErrorCode = ExtrActivateEmailFilter(DocumentMemory, StopMemory);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      printf("\nMail filter activated. \n");
    }
  }

  /*
  **    ask user about character coding for input text
  */

  /*
  **    English, French, German, or Spanish input character code
  */

  if ((LanguageID != LAN_JAPANESE) && (LanguageID != LAN_KOREAN)) printf(
    "\n"
    "Enter the character coding that is used in the file.\n"
    "\n"
    " 0 = ISO Latin-1    The default encoding, also known as ISO-8859-1.\n"
    " 1 = MS-DOS         Also known as MS-DOS Code Page 437.\n"
    " 2 = Unicode UCS2   Unicode double-byte encoding, in native byte order.\n"
    "\n");

  /*
  **    Japanese input character code
  */

  else if (LanguageID == LAN_JAPANESE) printf(
    "\n"
    "Enter the character coding that is used in the file.\n"
    "\n"
    " 2 = Unicode UCS2   Unicode double-byte encoding, in native byte order.\n"
    " 3 = Shift-JIS      SJIS, MS-Kanji, Code Page 932.\n"
    " 4 = JIS            New, Old, NEC, ISO-2022-jp.\n"
    " 5 = EUC-JP         Packed format, 1 and 2 bytes.\n"
    "\n"
    "Note: Japanese supports JIS X 0208-1990 + ASCII + halfwidth katakana.\n"
    "\n");

  /*
  **    Korean input character code
  */

  else if (LanguageID == LAN_KOREAN) printf(
    "\n"
    "Enter the character coding that is used in the file.\n"
    "\n"
    " 2 = Unicode UCS2   Unicode double-byte encoding, in native byte order.\n"
    " 6 = KSC            KS X 1001, EUC-KR, Code Page 949 (for Windows)\n" 
    " 7 = Johap          Old KS C 5601:1992 and KSSM\n"
    "\n"
    "Note: KSC represents Precomposed Syllable Encoding Method,\n" 
    "      including KS X 1001 (KS C 5657:1987), EUC_KR and Code Page 949.\n"
    "      JOHAP represents Syllable Composition Encoding Method,\n"
    "      including KSSM and KS C 5601:1992.\n"
    "\n");

  /*
  **     ask user for input character code
  */

  printf("Encoding > ");
  fgets(code_string, IN_LEN, stdin);
  code_string[strcspn(code_string, "\n\r")] = '\0';

  if ((code_string[0] == '\0') && (LanguageID == LAN_JAPANESE)) {
    printf("\nEXTRACTOR will use Shift-JIS.\n");
    InCharCodeID = ENC_SJIS;
  }
  else if ((code_string[0] == '\0') && (LanguageID == LAN_KOREAN)) {
    printf("\nEXTRACTOR will use KSC.\n");
    InCharCodeID = ENC_KSC;
  }
  else if (code_string[0] == '\0') {
    printf("\nEXTRACTOR will use ISO Latin-1.\n");
    InCharCodeID = ENC_LAT;
  }
  else {
    ErrorCode = sscanf(code_string, "%d", &InCharCodeID);
    if (ErrorCode != 1) {
      printf("\n\nERROR: your input is not a valid number.\n\n");
      return 0;
    }
  }

  if (LanguageID == LAN_JAPANESE) {
    if ((InCharCodeID < ENC_UNI) || (InCharCodeID > ENC_EUC)) {
      printf("\n\nERROR: the input character code is not suitable for Japanese.\n\n");
      return 0;
    }
  }
  else if (LanguageID == LAN_KOREAN) {
    if ((InCharCodeID != ENC_UNI) && 
        (InCharCodeID != ENC_KSC) && 
        (InCharCodeID != ENC_JOHAP)) {
      printf("\n\nERROR: the input character code is not suitable for Korean.\n\n");
      return 0;
    }
  }
  else if ((InCharCodeID < ENC_LAT) || (InCharCodeID > ENC_UNI)) {
    printf("\n\nERROR: the input character code is not suitable.\n\n");
    return 0;
  }

  ErrorCode = ExtrSetInputCode(DocumentMemory, InCharCodeID);
  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }
  printf("\nEncoding is set to %d. \n", InCharCodeID);

  /*
  **    ask user how many keyphrases are desired
  */

  printf(
    "\n"
    "How many keyphrases would you like EXTRACTOR to generate? Enter a blank line\n"
    "for the default value (7) or enter a number from 3 to 30. On average, EXTRACTOR\n"
    "will generate the requested number of phrases, but the actual number for any\n"
    "given document may be slightly below or above the requested number, depending\n"
    "mainly on the length of the input document.\n"
    "\n");

  printf("Desired Number of Phrases > ");
  fgets(num_phrase_string, IN_LEN, stdin);
  num_phrase_string[strcspn(num_phrase_string, "\n\r")] = '\0';

  if (num_phrase_string[0] == '\0') {
    printf("\nThe desired number of phrases is set to seven (7). \n");
  }
  else {
    ErrorCode = sscanf(num_phrase_string, "%lf", &DesiredNumber);
    if (ErrorCode != 1) {
      printf("\n\nERROR: your input is not a valid number.\n\n");
      return 0;
    }
    ErrorCode = ExtrSetNumberPhrases(DocumentMemory, DesiredNumber);
    if (ErrorCode > 0) {
      ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
      printf("\n\n%s\n\n", ErrorMessage);
      return 0;
    }
    printf("\nThe desired number of phrases is set to %.1f. \n", DesiredNumber);
  }

  /*
  **    ask the user whether to send the output to the standard output
  **    or an HTML file
  */

  printf(
    "\n"
    "Would you like the keyphrases and highlights to be sent to the standard output\n"
    "or would you like to generate an HTML file, \"%s\"? Enter a blank\n"
    "line for the default value (standard output) or enter \"f\" for an HTML file.\n"
    "If the language is Japanese or Korean and your operating system does not\n"
    "support the display of Japanese or Korean characters, then you should enter\n"
    "\"f\" for an HTML file.\n"
    "\n", HTML_FILE_NAME);

  printf("Standard Output or HTML > ");
  fgets(output_device, IN_LEN, stdin);
  output_device[strcspn(output_device, "\n\r")] = '\0';

  if (output_device[0] == '\0') {
    printf("\nKeyphrases and highlights will be sent to the standard output.\n");
    OutputDevice = 0;
  }
  else {
    printf("\nKeyphrases and highlights will be sent to \"%s\".\n", HTML_FILE_NAME);
    OutputDevice = 1;
  }

  /*
  **    set the output character code for the standard output
  **
  **
  **    Platform  Language                           Output Character Code
  **    ---------------------------------------------------------------------------
  **    Windows   English, French, German, Spanish   1 (MS-DOS Code Page 437)
  **    Windows   Japanese                           3 (SJIS, MS-Kanji, Code Page 932)
  **    Windows   Korean                             6 (KS X 1001,Code Page 949)
  **    Unix      English, French, German, Spanish   0 (ISO Latin-1, ISO-8859-1)
  **    Unix      Japanese                           5 (EUC-JP, Extended Unix Code)
  **    Unix      Korean                             6 (EUC-KR, Extended Unix Code)
  **    
  **    Note: KSC represents KS X 1001, EUC-KR and Code Page 949
  */

  if (OutputDevice == 0) {

    if (Windows == 1) {

      if (LanguageID == LAN_JAPANESE) {
        OutCharCodeID = ENC_SJIS;
        ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
        if (ErrorCode > 0) {
          ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          printf("\n\n%s\n\n", ErrorMessage);
          return 0;
        }
        printf(
          "\n"
          "Since the operating system is Windows and the language is Japanese, the\n"
          "output character code has been set to SJIS (MS-Kanji, Code Page 932).\n"
          "You will not be able to view the following keyphrases unless your system\n"
          "can handle SJIS output.\n"
          "\n");
      }

      else if (LanguageID == LAN_KOREAN) {
        OutCharCodeID = ENC_KSC;
        ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
        if (ErrorCode > 0) {
          ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          printf("\n\n%s\n\n", ErrorMessage);
          return 0;
        }
        printf(
          "\n"
          "Since the operating system is Windows and the language is Korean, the\n"
          "output character code has been set KSC (KS X 1001, Code Page 949).\n"
          "You will not be able to view the following keyphrases unless your system\n"
          "can handle KSC output.\n"
          "\n");
      }

      else {
        OutCharCodeID = ENC_DOS;
        ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
        if (ErrorCode > 0) {
          ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          printf("\n\n%s\n\n", ErrorMessage);
          return 0;
        }
        printf(
          "\n"
          "Since the operating system is Windows and the language is English, French,\n"
          "German, or Spanish, the output character code has been set to MS-DOS Code\n"
          "Page 437.\n"
          "\n");
      }
    }

    if (Windows == 0) {

      if (LanguageID == LAN_JAPANESE) {
        OutCharCodeID = ENC_EUC;
        ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
        if (ErrorCode > 0) {
          ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          printf("\n\n%s\n\n", ErrorMessage);
          return 0;
        }
        printf(
          "\n"
          "Since the operating system is Unix and the language is Japanese, the\n"
          "output character code has been set to EUC-JP (Extended Unix Code).\n"
          "You will not be able to view the following keyphrases unless your system\n"
          "can handle EUC-JP output.\n"
          "\n");
      }

      else if (LanguageID == LAN_KOREAN) {
        OutCharCodeID = ENC_KSC;
        ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
        if (ErrorCode > 0) {
          ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          printf("\n\n%s\n\n", ErrorMessage);
          return 0;
        }
        printf(
          "\n"
          "Since the operating system is Unix and the language is Korean, the\n"
          "output character code has been set EUC-KR (Extended Unix Code, KS X 2901).\n"
          "You will not be able to view the following keyphrases unless your system\n"
          "can handle EUC-KR output.\n"
          "\n");
      }

      else {
        OutCharCodeID = ENC_LAT;
        ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
        if (ErrorCode > 0) {
          ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          printf("\n\n%s\n\n", ErrorMessage);
          return 0;
        }
        printf(
          "\n"
          "Since the operating system is Unix and the language is English, French,\n"
          "German, or Spanish, the output character code has been set to ISO Latin-1\n"
          "(ISO-8859-1).\n"
          "\n");
      }
    }
  }

  /*
  **    set the output character code for HTML
  **
  **
  **    Platform   Language                           Output Character Code
  **    ---------------------------------------------------------------------------
  **    All        Japanese                           3 (SJIS, MS-Kanji, Code Page 932)
  **    All        Korean                             6 (KSC, EUC-KR, Code Page 949)
  **    All        English, French, German, Spanish   0 (ISO Latin-1, ISO-8859-1)
  */

  if (OutputDevice == 1) {

    if (LanguageID == LAN_JAPANESE) {
      OutCharCodeID = ENC_SJIS;
      ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      printf(
        "\n"
        "Since the language is Japanese, the output character code has been set\n"
        "to SJIS (MS-Kanji, Code Page 932). You will not be able to view the HTML\n"
        "file properly unless your web browser has Japanese fonts installed.\n"
        "\n");
    }

    else if (LanguageID == LAN_KOREAN) {
      OutCharCodeID = ENC_KSC;
      ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      printf(
        "\n"
        "Since the language is Korean, the output character code has been set\n"
        "to KSC (EUC-KR, Code Page 949).You will not be able to view the HTML\n"
        "file properly unless your web browser has Korean fonts installed.\n"
        "\n");
    }

    else {
      OutCharCodeID = ENC_LAT;
      ErrorCode  = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      printf(
        "\n"
        "Since the language is English, French, German, or Spanish, the output\n"
        "character code has been set to ISO Latin-1 (ISO-8859-1).\n"
        "\n");
    }
  }

  /*
  **    activate highlights
  */

  ErrorCode = ExtrActivateHighlights(DocumentMemory);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    set highlights type
  **
  **    --------------------------------------------------------------------
  **     HighlightType    Description
  **    --------------------------------------------------------------------
  **      0  <00000000>   - DEFAULT
  **                      - highlights in SAME ORDER AS KEYPHRASES
  **                      - duplicate highlights left, to preserve simple
  **                        mapping between highlights and keyphrases
  **                      - no HTML markup
  **                      - try to trim longer sentences into shorter phrases,
  **                        using a variety of heuristics
  **
  **      1  <000????1>   - REMOVE DUPLICATE HIGHLIGHTS
  **
  **      2  <000???1?>   - ORDER OF APPEARANCE IN TEXT
  **
  **      4  <000??1??>   - FULL SENTENCES
  **
  **      8  <000?1???>   - MARK-UP KEYPHRASE
  **
  **      16 <0001????>   - MARK-UP THE REMOVED PARTS OF FULL SENTENCE
  **         <0001?1??>     (Selecting this flag, will automatically select
  **                        flag 4)
  **
  **    --------------------------------------------------------------------
  **
  **    These types can be added; for example, type "5" is the combination
  **    of types "1" and "4" (duplicates removed, full sentences).
  **
  **
  **    NOTE: <00000011> is intended to represent a string of 8 bits
  **          (one byte). <00000011> encodes an unsigned integer value
  **          of "3". On the other hand, 0x00000011 represents a
  **          string of 8 nybbles (one nybble equals four bits), or
  **          32 bits. 0x00000011 encodes an unsigned integer value 
  **          of "17".
  */

  if (OutputDevice == 0) {
    HighlightType = 1 + 2 + 4;
    ErrorCode = ExtrSetHighlightType(DocumentMemory, HighlightType);
  }

  if (OutputDevice == 1) {
    HighlightType = 1 + 2 + 4 + 8 + 16;
    ErrorCode = ExtrSetHighlightType(DocumentMemory, HighlightType);
  }

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    ask the user for a go phrase
  */

  printf(
    "\n"
    "If there is a particular word or phrase that interests you, EXTRACTOR can look\n"
    "for that word or phrase in the input document. If it is found, it will be\n"
    "treated as a key word or key phrase. Enter a word or phrase, or enter a blank\n"
    "line if there is no special word or phrase.\n"
    "\n");

  printf("Special word or phrase > ");
  fgets(go_phrase, IN_LEN, stdin);
  go_phrase[strcspn(go_phrase, "\n\r")] = '\0';

  if (go_phrase[0] == '\0') {
    printf("\nNo special word or phrase was entered.\n");
  }
  else {

    /*
    **    the user entered a go phrase
    **
    **    now we need to know what encoding was used
    */

    printf("\nEXTRACTOR will look for \"%s\".\n", go_phrase);
    printf(
      "\n"
      "To interpret this phrase correctly, EXTRACTOR needs to know the encoding of the\n"
      "phrase. Enter a blank line to use the same encoding as the input document (%d).\n"
      "\n", InCharCodeID);
    if ((LanguageID != LAN_JAPANESE) && (LanguageID != LAN_KOREAN)) printf(
      " 0 = ISO Latin-1    The default encoding, also known as ISO-8859-1.\n"
      " 1 = MS-DOS         Also known as MS-DOS Code Page 437.\n"
      " 2 = Unicode UCS2   Unicode double-byte encoding, in native byte order.\n"
      "\n");
    else if (LanguageID == LAN_JAPANESE) printf(
      " 2 = Unicode UCS2   Unicode double-byte encoding, in native byte order.\n"
      " 3 = Shift-JIS      SJIS, MS-Kanji, Code Page 932.\n"
      " 4 = JIS            New, Old, NEC, ISO-2022-jp.\n"
      " 5 = EUC-JP         Packed format, 1 and 2 bytes.\n"
      "\n");
    else if (LanguageID == LAN_KOREAN) printf(
      " 2 = Unicode UCS2   Unicode double-byte encoding, in native byte order.\n"
      " 6 = KSC            KS X 1001, EUC-KR, Code Page 949 (for Windows)\n" 
      " 7 = Johap          Old KS C 5601:1992 and KSSM\n"
      "\n");
    printf("Encoding > ");
    fgets(code_string, IN_LEN, stdin);
    code_string[strcspn(code_string, "\n\r")] = '\0';
    if (code_string[0] != '\0') {
      ErrorCode = sscanf(code_string, "%d", &InCharCodeID);
      if (ErrorCode != 1) {
        printf("\n\nERROR: your input is not a valid number.\n\n");
        return 0;
      }
    }
    printf("\nEXTRACTOR will use %d.\n", InCharCodeID);

    /*
    **    now we know the code, so we can add the go phrase to the list
    **    of go phrases
    */

    if (LanguageID == LAN_UNKNOWN) {

      /*
      **    since the language is unknown, it could be English,
      **    French, German, or Spanish; so we add the go phrase
      **    to all four languages
      */

      ErrorCode = ExtrAddGoPhrase(StopMemory, LAN_ENGLISH, 
                    InCharCodeID, go_phrase, MATCH_WHOLE_WORD);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      ErrorCode = ExtrAddGoPhrase(StopMemory, LAN_FRENCH, 
                    InCharCodeID, go_phrase, MATCH_WHOLE_WORD);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      ErrorCode = ExtrAddGoPhrase(StopMemory, LAN_GERMAN, 
                    InCharCodeID, go_phrase, MATCH_WHOLE_WORD);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      ErrorCode = ExtrAddGoPhrase(StopMemory, LAN_SPANISH, 
                    InCharCodeID, go_phrase, MATCH_WHOLE_WORD);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
    }
    else {

      /*
      **    since the language is known, add the go phrase
      **    for the selected language only
      */

      ErrorCode = ExtrAddGoPhrase(StopMemory, LanguageID, 
                    InCharCodeID, go_phrase, MATCH_WHOLE_WORD);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
    }
  }

  /*
  **    read the document buffer
  */

  t_start = time(NULL);

  while (! feof(text_file)) {
    BufferLength = fread(DocumentBuffer, sizeof(char), MAX_BUF_LEN, text_file);
    if (BufferLength == 0) break;
    ErrorCode = ExtrReadDocumentBuffer(DocumentMemory, StopMemory,
                  (void *) DocumentBuffer, BufferLength);
    if (ErrorCode > 0) {
      ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
      printf("\n\n%s\n\n", ErrorMessage);
      return 0;
    }
  }

  /*
  **    signal end of document
  */

  ErrorCode = ExtrSignalDocumentEnd(DocumentMemory, StopMemory);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  t_stop = time(NULL);

  /*
  **    close the input file
  */

  fclose(text_file);

  /*
  **    see what language was chosen for the document
  */

  ErrorCode = ExtrGetDocumentLanguage(DocumentMemory, &LanguageID);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  switch (LanguageID) {
    case  LAN_UNKNOWN :  printf("\nThe language is unknown.\n");
              break;
    case  LAN_ENGLISH :  printf("\nThe selected language was English.\n");
              break;
    case  LAN_FRENCH :   printf("\nThe selected language was French.\n");
              break;
    case  LAN_JAPANESE : printf("\nThe selected language was Japanese.\n");
              break;
    case  LAN_GERMAN :   printf("\nThe selected language was German.\n");
              break;
    case  LAN_SPANISH :  printf("\nThe selected language was Spanish.\n");
              break;
    case  LAN_KOREAN :   printf("\nThe selected language was Korean.\n");
              break;
  }

  /*
  **    get the phrase list size
  */

  ErrorCode = ExtrGetPhraseListSize(DocumentMemory, &PhraseListSize);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    see whether any keyphrases were generated
  */

  if (PhraseListSize == 0) {
    printf("\nNo keyphrases were extracted! \n\n");
  }

  /*
  **    if the phrase list size is greater than zero, then
  **    print the phrases to the standard output
  */

  if ((OutputDevice == 0) && (PhraseListSize > 0)) {
    printf("\nKeyphrases: \n\n");
    for (PhraseIndex = 0; PhraseIndex < PhraseListSize; PhraseIndex++) {
      ErrorCode = ExtrGetPhraseByIndex(DocumentMemory, PhraseIndex, 
                    (void **) &Phrase);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      ErrorCode = ExtrGetScoreByIndex(DocumentMemory, PhraseIndex, 
                    &Score);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      printf ("     %-40s     (%10.2f) \n", Phrase, Score);
    }
  }

  /*
  **    get the highlight list size
  */

  ErrorCode = ExtrGetHighlightListSize(DocumentMemory, &HighlightListSize);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    see whether any highlights were generated
  */

  if (HighlightListSize == 0) {
    printf("\nNo highlights were generated! \n\n");
  }

  /*
  **    if the highlight list size is greater than zero, then
  **    print the highlights to the standard output
  */

  if ((OutputDevice == 0) && (HighlightListSize > 0)) {
    printf("\nHighlights: \n\n");
    for (HighlightIndex = 0; HighlightIndex < HighlightListSize; HighlightIndex++) {
      ErrorCode = ExtrGetHighlightByIndex(DocumentMemory, HighlightIndex, 
                    (void **) &Highlight);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      if (strlen(Highlight) < 70)
        printf ("   - %s \n", Highlight);
      else
        printf ("   - %-.70s ...\n", Highlight);
    }
  }

  /*
  **    print run time
  */

  run_time = (int) difftime(t_stop, t_start);

  if (run_time < 1) 
    printf("\nEXTRACTOR took less than 1 second. \n");
  else if (run_time == 1) 
    printf("\nEXTRACTOR took 1 second. \n");
  else 
    printf("\nEXTRACTOR took %d seconds. \n", run_time);

  /*
  **    get the number of words that were read
  */

  PropID = PROP_WORDS;

  ErrorCode = ExtrGetDocumentProperties(DocumentMemory, PropID, &PropValue);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  printf("EXTRACTOR read %d words. \n\n", PropValue);

  /*
  **    print keyphrases and highlights to a file
  */

  /*
  **    HTML output
  **
  **    - output ISO Latin-1 (ISO-8859-1) for English, French, German,
  **      or Spanish
  **
  **    - output SJIS (Shift_JIS) for Japanese
  **
  **    - output KSC (EUC-KR, Code Page 949) for Korean
  */

  if (OutputDevice == 1) {

    out_file = fopen(HTML_FILE_NAME, "w");
    if (out_file == NULL) {
      printf("\n\nERROR: cannot open file \"%s\" for output. \n\n", HTML_FILE_NAME);
      return 0;
    }

    if (OutCharCodeID == ENC_LAT) fprintf(out_file, 
      "<HTML>\n"
      "<HEAD>\n"
      "<META http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\">\n"
      "</HEAD>\n"
      "<BODY>\n");
    if (OutCharCodeID == ENC_SJIS) fprintf(out_file, 
      "<HTML>\n"
      "<HEAD>\n"
      "<META http-equiv=\"Content-Type\" content=\"text/html; charset=Shift_JIS\">\n"
      "</HEAD>\n"
      "<BODY>\n");
    if (OutCharCodeID == ENC_KSC) fprintf(out_file, 
      "<HTML>\n"
      "<HEAD>\n"
      "<META http-equiv=\"Content-Type\" content=\"text/html; charset=EUC-KR\">\n"
      "</HEAD>\n"
      "<BODY>\n");

    fprintf(out_file, "\n\n<P>Source Document: %s </P>\n\n", file_name);
    fprintf(out_file, "\n<P>Keyphrases: </P>\n\n");
    fprintf(out_file, "<MENU>\n");

    for (PhraseIndex = 0; PhraseIndex < PhraseListSize; PhraseIndex++) {
      ErrorCode = ExtrGetPhraseByIndex(DocumentMemory, PhraseIndex, 
                    (void **) &Phrase);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      fprintf(out_file, "<LI> %s\n", Phrase);
    }

    if (PhraseListSize == 0) {
      fprintf(out_file, "<LI> No keyphrases were extracted! \n");
    }

    fprintf(out_file, "</MENU>\n");
    fprintf(out_file, "\n<P>Highlights: </P>\n\n");
    fprintf(out_file, "<MENU>\n");

    for (HighlightIndex = 0; HighlightIndex < HighlightListSize; HighlightIndex++) {
      ErrorCode = ExtrGetHighlightByIndex(DocumentMemory, HighlightIndex, 
                    (void **) &Highlight);
      if (ErrorCode > 0) {
        ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
        printf("\n\n%s\n\n", ErrorMessage);
        return 0;
      }
      fprintf(out_file, "<LI> %s\n", Highlight);
    }

    if (HighlightListSize == 0) {
      fprintf(out_file, "<LI> No highlights were extracted! \n");
    }

    fprintf(out_file, "</MENU>\n");
    fprintf(out_file,
      "</BODY>\n"
      "</HTML>\n");
    fclose(out_file);

    if (OutCharCodeID == ENC_LAT) printf(
      "\n"
      "The keyphrases and highlights have been written to the file \"%s\",\n"
      "using ISO Latin-1 (ISO-8859-1) character coding. This file can be viewed with\n"
      "most Web browsers.\n"
      "\n", HTML_FILE_NAME);
    if (OutCharCodeID == ENC_SJIS) printf(
      "\n"
      "The keyphrases and highlights have been written to the file \"%s\",\n"
      "using SJIS (MS-Kanji, Code Page 932) character coding. This file can be viewed\n"
      "with most Web browsers, if Japanese fonts are installed for the browser.\n"
      "\n", HTML_FILE_NAME);
    if (OutCharCodeID == ENC_KSC) printf(
      "\n"
      "The keyphrases and highlights have been written to the file \"%s\",\n"
      "using KSC (KS X 1001, EUC-KR, Code Page 949) character coding. This file can be\n"
      "viewed with most Web browsers, if Korean fonts are installed for the browser.\n"
      "\n", HTML_FILE_NAME);
  }

  /*
  **    clear the document memory
  */

  ErrorCode = ExtrClearDocumentMemory(DocumentMemory);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    clear the stop word and stop phrase lists
  */

  ErrorCode = ExtrClearStopMemory(StopMemory);

  if (ErrorCode > 0) {
    ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
    printf("\n\n%s\n\n", ErrorMessage);
    return 0;
  }

  /*
  **    end of main
  */

  return 0;
}

